# 垃圾信息分类检测
# SMSSpamCollection.csv 垃圾信息检测数据集
from keras.layers import SimpleRNN,Embedding,Dense,LSTM
from keras.models import Sequential
from keras.preprocessing.text import Tokenizer
from keras.preprocessing.sequence import pad_sequences
from keras_preprocessing import sequence
import numpy as np
from numpy.core.numeric import indices
from sklearn.metrics import confusion_matrix
import matplotlib.pyplot as plt
import pandas as pd

data = pd.read_csv("./drive/MyDrive/app/data/SMSSpamCollection", sep='\t')
print(data.info())
print(data.head(5))
texts = []
classes = []
for i in data.index:
    texts.append(data.loc[i].values[1])
    label = data.loc[i].values[0]
    if label == 'ham':
        classes.append(0)
    else:
        classes.append(1)
texts = np.asarray(texts)
classes = np.asarray(classes)
print("number of texts:",len(texts))
print("number of labels:",len(classes))

maxFeatures = 10000
maxLen = 500

trainData = int(len(texts)*0.8)
valiData = int(len(texts)-trainData)

tokenizer = Tokenizer()
tokenizer.fit_on_texts(texts)
sequences = tokenizer.texts_to_sequences(texts)
word_index = tokenizer.word_index
print("found {0} unique words".format(len(word_index)))
data = pad_sequences(sequences,maxlen=maxLen)
print("data shape:",data.shape)

indices = np.arange(data.shape[0])
np.random.shuffle(indices)
data = data[indices]
labels = classes[indices]
X_train = data[:trainData]
y_train = labels[:trainData]
X_test = data[trainData:]
y_test = labels[trainData:]

model = Sequential()
model.add(Embedding(maxFeatures,30))
model.add(LSTM(32))
model.add(Dense(1,activation='sigmoid'))
model.compile(optimizer='rmsprop',loss='binary_crossentropy',metrics=['acc'])
run = model.fit(X_train,y_train,epochs=10,batch_size=60,validation_split=0.2)
pred = model.predict_classes(X_test)

acc = model.evaluate(X_test,y_test)
proba_rnn = model.predict_proba(X_test,y_test)
print("Test loss is {0:.2f} accuracy is {1:%.2f}".format(acc[0],acc[1]))
print(confusion_matrix(pred,y_test))


